2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
3 # http://www.mediawiki.org/
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc.,
17 # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 # http://www.gnu.org/copyleft/gpl.html
20 # This script generates UniNormalData.inc from the Unicode Character Database
21 # and supplementary files.
23 require_once 'UtfNormalUtil.php';
25 $in = fopen("DerivedNormalizationProps.txt", "rt" );
27 print "Can't open DerivedNormalizationProps.txt for reading.\n";
28 print "If necessary, fetch this file from the internet:\n";
29 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
32 print "Initializing normalization quick check tables...\n";
34 while( false !== ($line = fgets( $in ) ) ) {
35 if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
36 list( $junk, $first, $last, $prop, $value ) = $matches;
37 #print "$first $last $prop $value\n";
38 if( !$last ) $last = $first;
39 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++
) {
40 $char = codepointToUtf8( $i );
41 $checkNFC[$char] = $value;
47 $in = fopen("CompositionExclusions.txt", "rt" );
49 print "Can't open CompositionExclusions.txt for reading.\n";
50 print "If necessary, fetch this file from the internet:\n";
51 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
55 while( false !== ($line = fgets( $in ) ) ) {
56 if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
57 $codepoint = $matches[1];
58 $source = codepointToUtf8( hexdec( $codepoint ) );
59 $exclude[$source] = true;
64 $in = fopen("UnicodeData.txt", "rt" );
66 print "Can't open UnicodeData.txt for reading.\n";
67 print "If necessary, fetch this file from the internet:\n";
68 print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
72 $compatibilityDecomp = array();
73 $canonicalDecomp = array();
74 $canonicalComp = array();
75 $combiningClass = array();
80 print "Reading character definitions...\n";
81 while( false !== ($line = fgets( $in ) ) ) {
82 $columns = split(';', $line);
83 $codepoint = $columns[0];
85 $canonicalCombiningClass = $columns[3];
86 $decompositionMapping = $columns[5];
88 $source = codepointToUtf8( hexdec( $codepoint ) );
90 if( $canonicalCombiningClass != 0 ) {
91 $combiningClass[$source] = IntVal( $canonicalCombiningClass );
94 if( $decompositionMapping === '' ) continue;
95 if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
96 # Compatibility decomposition
98 $decompositionMapping = $matches[2];
105 $dest = hexSequenceToUtf8( $decompositionMapping );
107 $compatibilityDecomp[$source] = $dest;
109 $canonicalDecomp[$source] = $dest;
110 if( empty( $exclude[$source] ) ) {
111 $canonicalComp[$dest] = $source;
114 #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
118 print "Recursively expanding canonical mappings...\n";
121 while( $changed > 0 ) {
122 print "pass $pass\n";
124 foreach( $canonicalDecomp as $source => $dest ) {
125 $newDest = preg_replace_callback(
126 '/([\xc0-\xff][\x80-\xbf]+)/',
129 if( $newDest === $dest ) continue;
131 $canonicalDecomp[$source] = $newDest;
136 print "Recursively expanding compatibility mappings...\n";
139 while( $changed > 0 ) {
140 print "pass $pass\n";
142 foreach( $compatibilityDecomp as $source => $dest ) {
143 $newDest = preg_replace_callback(
144 '/([\xc0-\xff][\x80-\xbf]+)/',
147 if( $newDest === $dest ) continue;
149 $compatibilityDecomp[$source] = $newDest;
154 print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
156 $out = fopen("UtfNormalData.inc", "wt");
158 $serCombining = escapeSingleString( serialize( $combiningClass ) );
159 $serComp = escapeSingleString( serialize( $canonicalComp ) );
160 $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
161 $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
162 $outdata = "<" . "?php
163 # This file was automatically generated -- do not edit!
164 # Run UtfNormalGenerate.php to create this file again (make clean && make)
166 global \$utfCombiningClass, \$utfCanonicalComp, \$utfCanonicalDecomp;
167 \$utfCombiningClass = unserialize( '$serCombining' );
168 \$utfCanonicalComp = unserialize( '$serComp' );
169 \$utfCanonicalDecomp = unserialize( '$serCanon' );
170 \$utfCheckNFC = unserialize( '$serCheckNFC' );
172 fputs( $out, $outdata );
174 print "Wrote out UtfNormalData.inc\n";
176 print "Can't create file UtfNormalData.inc\n";
181 $out = fopen("UtfNormalDataK.inc", "wt");
183 $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
184 $outdata = "<" . "?php
185 # This file was automatically generated -- do not edit!
186 # Run UtfNormalGenerate.php to create this file again (make clean && make)
188 global \$utfCompatibilityDecomp;
189 \$utfCompatibilityDecomp = unserialize( '$serCompat' );
191 fputs( $out, $outdata );
193 print "Wrote out UtfNormalDataK.inc\n";
196 print "Can't create file UtfNormalDataK.inc\n";
202 function callbackCanonical( $matches ) {
203 global $canonicalDecomp;
204 if( isset( $canonicalDecomp[$matches[1]] ) ) {
205 return $canonicalDecomp[$matches[1]];
210 function callbackCompat( $matches ) {
211 global $compatibilityDecomp;
212 if( isset( $compatibilityDecomp[$matches[1]] ) ) {
213 return $compatibilityDecomp[$matches[1]];